{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading train and test set\n",
      "Datasets succ. loaded\n",
      "BERT_MODEL dir uncased_L-12_H-768_A-12\n",
      "models/uncased_L-12_H-768_A-12\n",
      "Loaded auxiliary scripts\n",
      "train_examples object loaded\n",
      "Steps calculated\n",
      "Please wait..., loading train words in model\n",
      "INFO:tensorflow:Writing example 0 of 159571\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train\n",
      "INFO:tensorflow:tokens: [CLS] explanation why the edit ##s made under my user ##name hardcore metallic ##a fan were reverted ? they weren ' t van ##dal ##isms , just closure on some gas after i voted at new york dolls fa ##c . and please don ' t remove the template from the talk page since i ' m retired now . 89 . 205 . 38 . 27 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 7526 2339 1996 10086 2015 2081 2104 2026 5310 18442 13076 12392 2050 5470 2020 16407 1029 2027 4694 1005 1056 3158 9305 22556 1010 2074 8503 2006 2070 3806 2044 1045 5444 2012 2047 2259 14421 6904 2278 1012 1998 3531 2123 1005 1056 6366 1996 23561 2013 1996 2831 3931 2144 1045 1005 1049 3394 2085 1012 6486 1012 16327 1012 4229 1012 2676 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train\n",
      "INFO:tensorflow:tokens: [CLS] d ' aw ##w ! he matches this background colour i ' m seemingly stuck with . thanks . ( talk ) 21 : 51 , january 11 , 2016 ( utc ) [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1040 1005 22091 2860 999 2002 3503 2023 4281 6120 1045 1005 1049 9428 5881 2007 1012 4283 1012 1006 2831 1007 2538 1024 4868 1010 2254 2340 1010 2355 1006 11396 1007 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train\n",
      "INFO:tensorflow:tokens: [CLS] hey man , i ' m really not trying to edit war . it ' s just that this guy is constantly removing relevant information and talking to me through edit ##s instead of my talk page . he seems to care more about the format ##ting than the actual info . [SEP]\n",
      "INFO:tensorflow:input_ids: 101 4931 2158 1010 1045 1005 1049 2428 2025 2667 2000 10086 2162 1012 2009 1005 1055 2074 2008 2023 3124 2003 7887 9268 7882 2592 1998 3331 2000 2033 2083 10086 2015 2612 1997 2026 2831 3931 1012 2002 3849 2000 2729 2062 2055 1996 4289 3436 2084 1996 5025 18558 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train\n",
      "INFO:tensorflow:tokens: [CLS] \" more i can ' t make any real suggestions on improvement - i wondered if the section statistics should be later on , or a sub ##section of \" \" types of accidents \" \" - i think the references may need tidy ##ing so that they are all in the exact same format ie date format etc . i can do that later on , if no - one else does first - if you have any preferences for format ##ting style on references or want to do it yourself please let me know . there appears to be a back ##log on articles for review so i guess there may be a delay until a reviewer turns up . it ' s listed [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1000 2062 1045 2064 1005 1056 2191 2151 2613 15690 2006 7620 1011 1045 4999 2065 1996 2930 6747 2323 2022 2101 2006 1010 2030 1037 4942 29015 1997 1000 1000 4127 1997 13436 1000 1000 1011 1045 2228 1996 7604 2089 2342 29369 2075 2061 2008 2027 2024 2035 1999 1996 6635 2168 4289 29464 3058 4289 4385 1012 1045 2064 2079 2008 2101 2006 1010 2065 2053 1011 2028 2842 2515 2034 1011 2065 2017 2031 2151 18394 2005 4289 3436 2806 2006 7604 2030 2215 2000 2079 2009 4426 3531 2292 2033 2113 1012 2045 3544 2000 2022 1037 2067 21197 2006 4790 2005 3319 2061 1045 3984 2045 2089 2022 1037 8536 2127 1037 12027 4332 2039 1012 2009 1005 1055 3205 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train\n",
      "INFO:tensorflow:tokens: [CLS] you , sir , are my hero . any chance you remember what page that ' s on ? [SEP]\n",
      "INFO:tensorflow:input_ids: 101 2017 1010 2909 1010 2024 2026 5394 1012 2151 3382 2017 3342 2054 3931 2008 1005 1055 2006 1029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:Writing example 10000 of 159571\n",
      "INFO:tensorflow:Writing example 20000 of 159571\n",
      "INFO:tensorflow:Writing example 30000 of 159571\n",
      "INFO:tensorflow:Writing example 40000 of 159571\n",
      "INFO:tensorflow:Writing example 50000 of 159571\n",
      "INFO:tensorflow:Writing example 60000 of 159571\n",
      "INFO:tensorflow:Writing example 70000 of 159571\n",
      "INFO:tensorflow:Writing example 80000 of 159571\n",
      "INFO:tensorflow:Writing example 90000 of 159571\n",
      "INFO:tensorflow:Writing example 100000 of 159571\n",
      "INFO:tensorflow:Writing example 110000 of 159571\n",
      "INFO:tensorflow:Writing example 120000 of 159571\n",
      "INFO:tensorflow:Writing example 130000 of 159571\n",
      "INFO:tensorflow:Writing example 140000 of 159571\n",
      "INFO:tensorflow:Writing example 150000 of 159571\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_features loaded\n",
      "Please wait..., loading test words in model\n",
      "INFO:tensorflow:Writing example 0 of 153164\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test\n",
      "INFO:tensorflow:tokens: [CLS] yo bitch ja rule is more su ##cc ##es ##ful then you ' ll ever be what ##s up with you and hating you sad mo ##fu ##ck ##as . . . i should bitch slap ur pet ##hed ##ic white faces and get you to kiss my ass you guys sick ##en me . ja rule is about pride in da music man . don ##t di ##ss that shit on him . and nothin is wrong bei ##n like tu ##pac he was a brother too . . . fuck ##in white boys get things right next time . , [SEP]\n",
      "INFO:tensorflow:input_ids: 101 10930 7743 14855 3627 2003 2062 10514 9468 2229 3993 2059 2017 1005 2222 2412 2022 2054 2015 2039 2007 2017 1998 22650 2017 6517 9587 11263 3600 3022 1012 1012 1012 1045 2323 7743 14308 24471 9004 9072 2594 2317 5344 1998 2131 2017 2000 3610 2026 4632 2017 4364 5305 2368 2033 1012 14855 3627 2003 2055 6620 1999 4830 2189 2158 1012 2123 2102 4487 4757 2008 4485 2006 2032 1012 1998 24218 2003 3308 21388 2078 2066 10722 19498 2002 2001 1037 2567 2205 1012 1012 1012 6616 2378 2317 3337 2131 2477 2157 2279 2051 1012 1010 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test\n",
      "INFO:tensorflow:tokens: [CLS] = = from rfc = = the title is fine as it is , im ##o . [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1027 1027 2013 14645 1027 1027 1996 2516 2003 2986 2004 2009 2003 1010 10047 2080 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test\n",
      "INFO:tensorflow:tokens: [CLS] \" = = sources = = * za ##we ashton on lap ##land — / \" [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1000 1027 1027 4216 1027 1027 1008 23564 8545 13772 2006 5001 3122 1517 1013 1000 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test\n",
      "INFO:tensorflow:tokens: [CLS] : if you have a look back at the source , the information i updated was the correct form . i can only guess the source hadn ' t updated . i shall update the information once again but thank you for your message . [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1024 2065 2017 2031 1037 2298 2067 2012 1996 3120 1010 1996 2592 1045 7172 2001 1996 6149 2433 1012 1045 2064 2069 3984 1996 3120 2910 1005 1056 7172 1012 1045 4618 10651 1996 2592 2320 2153 2021 4067 2017 2005 2115 4471 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test\n",
      "INFO:tensorflow:tokens: [CLS] i don ' t anonymous ##ly edit articles at all . [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1045 2123 1005 1056 10812 2135 10086 4790 2012 2035 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: [0 0 0 0 0 0] (id = 32)\n",
      "INFO:tensorflow:Writing example 10000 of 153164\n",
      "INFO:tensorflow:Writing example 20000 of 153164\n",
      "INFO:tensorflow:Writing example 30000 of 153164\n",
      "INFO:tensorflow:Writing example 40000 of 153164\n",
      "INFO:tensorflow:Writing example 50000 of 153164\n",
      "INFO:tensorflow:Writing example 60000 of 153164\n",
      "INFO:tensorflow:Writing example 70000 of 153164\n",
      "INFO:tensorflow:Writing example 80000 of 153164\n",
      "INFO:tensorflow:Writing example 90000 of 153164\n",
      "INFO:tensorflow:Writing example 100000 of 153164\n",
      "INFO:tensorflow:Writing example 110000 of 153164\n",
      "INFO:tensorflow:Writing example 120000 of 153164\n",
      "INFO:tensorflow:Writing example 130000 of 153164\n",
      "INFO:tensorflow:Writing example 140000 of 153164\n",
      "INFO:tensorflow:Writing example 150000 of 153164\n",
      "test_features loaded\n",
      "loading X_t and y_train\n",
      "loaded\n",
      "loading X_te\n",
      "X_te loaded\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# # Improved LSTM baseline\n",
    "# \n",
    "# This kernel is a somewhat improved version of [Keras - Bidirectional LSTM baseline](https://www.kaggle.com/CVxTz/keras-bidirectional-lstm-baseline-lb-0-051) along with some additional documentation of the steps. (NB: this notebook has been re-run on the new test set.)\n",
    "\n",
    "# In[1]:\n",
    "\n",
    "\n",
    "import sys, os, re, csv, codecs, numpy as np, pandas as pd\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D\n",
    "from keras.models import Model\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "\n",
    "\n",
    "# We include the GloVe word vectors in our input files. To include these in your kernel, simple click 'input files' at the top of the notebook, and search 'glove' in the 'datasets' section.\n",
    "\n",
    "# In[2]:\n",
    "\n",
    "\n",
    "path = 'input/'\n",
    "comp = 'jigsaw-toxic-comment-classification-challenge/'\n",
    "#EMBEDDING_FILE=f'{path}glove6b50d/glove.6B.50d.txt'\n",
    "TRAIN_DATA_FILE=f'{path}{comp}train.csv'\n",
    "TEST_DATA_FILE=f'{path}{comp}test.csv'\n",
    "\n",
    "\n",
    "# Set some basic config parameters:\n",
    "\n",
    "# In[3]:\n",
    "\n",
    "\n",
    "#embed_size = 50 # how big is each word vector\n",
    "embed_size = 768 # how big is each word vector\n",
    "max_features = 30522 # how many unique words to use (i.e num rows in embedding vector)\n",
    "maxlen = 128 # max number of words in a comment to use\n",
    "\n",
    "\n",
    "# Read in our data and replace missing values:\n",
    "\n",
    "# In[4]:\n",
    "\n",
    "\n",
    "train_df = pd.read_csv(TRAIN_DATA_FILE)\n",
    "test_df = pd.read_csv(TEST_DATA_FILE)\n",
    "\n",
    "\n",
    "print('Loading train and test set')\n",
    "list_sentences_train = train_df[\"comment_text\"].fillna(\"_na_\").values\n",
    "list_classes = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "y_train = train_df[list_classes].values\n",
    "list_sentences_test = test_df[\"comment_text\"].fillna(\"_na_\").values\n",
    "print('Datasets succ. loaded')\n",
    "'''\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train, test = train_test_split(train_df, test_size = 0.1, random_state=42)\n",
    "\n",
    "train_lines, train_labels = train.comment_text.values, train[list_classes].values\n",
    "test_lines, test_labels = test.comment_text.values, test[list_classes].values\n",
    "'''\n",
    "\n",
    "train_lines = list_sentences_train\n",
    "test_lines = list_sentences_test\n",
    "train_labels = y_train\n",
    "\n",
    "\n",
    "# In[5]:\n",
    "\n",
    "\n",
    "#list_sentences_test.shape\n",
    "\n",
    "\n",
    "# In[13]:\n",
    "\n",
    "\n",
    "#wget https://raw.githubusercontent.com/google-research/bert/master/modeling.py\n",
    "#wget https://raw.githubusercontent.com/google-research/bert/master/optimization.py\n",
    "#wget https://raw.githubusercontent.com/google-research/bert/master/run_classifier.py\n",
    "#wget https://raw.githubusercontent.com/google-research/bert/master/tokenization.py \n",
    "#wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip\n",
    "\n",
    "\n",
    "# In[6]:\n",
    "\n",
    "\n",
    "BERT_MODEL = 'uncased_L-12_H-768_A-12'\n",
    "BERT_PRETRAINED_DIR = 'models/uncased_L-12_H-768_A-12'\n",
    "print('BERT_MODEL dir',BERT_MODEL)\n",
    "print(BERT_PRETRAINED_DIR)\n",
    "\n",
    "# In[7]:\n",
    "\n",
    "\n",
    "import modeling\n",
    "import optimization\n",
    "import run_classifier\n",
    "import tokenization\n",
    "import tensorflow as tf\n",
    "\n",
    "print('Loaded auxiliary scripts')\n",
    "\n",
    "def create_examples(lines, set_type, labels=None):\n",
    "#Generate data for the BERT model\n",
    "    guid = f'{set_type}'\n",
    "    examples = []\n",
    "    if guid == 'train':\n",
    "        for line, label in zip(lines, labels):\n",
    "            text_a = line\n",
    "            label = str(label)\n",
    "            examples.append(\n",
    "              run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label))\n",
    "    else:\n",
    "        for line in lines:\n",
    "            text_a = line\n",
    "            label = '[0 0 0 0 0 0]'\n",
    "            examples.append(\n",
    "              run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label))\n",
    "    return examples\n",
    "\n",
    "# Model Hyper Parameters\n",
    "tf.random.set_random_seed(49)\n",
    "\n",
    "TRAIN_BATCH_SIZE = 32\n",
    "EVAL_BATCH_SIZE = 8\n",
    "LEARNING_RATE = 2e-5\n",
    "NUM_TRAIN_EPOCHS = 3.0\n",
    "WARMUP_PROPORTION = 0.1\n",
    "MAX_SEQ_LENGTH = 128\n",
    "# Model configs\n",
    "SAVE_CHECKPOINTS_STEPS = 1000 #if you wish to finetune a model on a larger dataset, use larger interval\n",
    "# each checpoint weights about 1,5gb\n",
    "ITERATIONS_PER_LOOP = 1000\n",
    "NUM_TPU_CORES = 8\n",
    "VOCAB_FILE = os.path.join(BERT_PRETRAINED_DIR, 'vocab.txt')\n",
    "CONFIG_FILE = os.path.join(BERT_PRETRAINED_DIR, 'bert_config.json')\n",
    "INIT_CHECKPOINT = os.path.join(BERT_PRETRAINED_DIR, 'bert_model.ckpt')\n",
    "DO_LOWER_CASE = BERT_MODEL.startswith('uncased')\n",
    "\n",
    "#label_list = ['0', '1']\n",
    "tokenizer = tokenization.FullTokenizer(vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE)\n",
    "train_examples = create_examples(train_lines, 'train', labels=train_labels)\n",
    "test_examples = create_examples(test_lines, 'test')\n",
    "\n",
    "print('train_examples object loaded')\n",
    "\n",
    "\n",
    "# In[9]:\n",
    "\n",
    "\n",
    "num_train_steps = int(\n",
    "    len(train_examples) / TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS)\n",
    "num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)\n",
    "\n",
    "print('Steps calculated')\n",
    "\n",
    "# In[10]:\n",
    "\n",
    "\n",
    "l = [] \n",
    "def return_l(n, l_r):\n",
    "    if n == 1:\n",
    "        l = l_r\n",
    "        return l_r\n",
    "    new_l = []\n",
    "    for t in l_r:\n",
    "        t.append(0)\n",
    "        new_l.append(t[:])\n",
    "        t.pop()\n",
    "        t.append(1)\n",
    "        new_l.append(t[:])\n",
    "        t.pop()\n",
    "    return return_l(n-1, new_l)\n",
    "\n",
    "def return_str(n, l_r):\n",
    "    if n == 1:\n",
    "        new_l = []\n",
    "        for t in l_r:\n",
    "            new_l.append(t +']')\n",
    "        l = new_l\n",
    "        return new_l\n",
    "    new_l = []\n",
    "    for t in l_r:\n",
    "        \n",
    "        new_l.append(t +' 0')\n",
    "        new_l.append(t +' 1')\n",
    "    return return_str(n-1, new_l)\n",
    "\n",
    "\n",
    "label_list_mult = return_str(6, ['[1','[0'])\n",
    "\n",
    "\n",
    "# In[12]:\n",
    "\n",
    "\n",
    "# Train the model.\n",
    "print('Please wait..., loading train words in model')\n",
    "train_features = run_classifier.convert_examples_to_features(train_examples, label_list_mult,  MAX_SEQ_LENGTH, tokenizer)\n",
    "print('train_features loaded')\n",
    "print('Please wait..., loading test words in model')\n",
    "test_features = run_classifier.convert_examples_to_features(test_examples, label_list_mult,  MAX_SEQ_LENGTH, tokenizer)\n",
    "print('test_features loaded')\n",
    "\n",
    "# In[13]:\n",
    "\n",
    "\n",
    "formOfList_label_list_mult = return_l(6, [[1], [0]])\n",
    "\n",
    "\n",
    "# In[16]:\n",
    "\n",
    "print('loading X_t and y_train')\n",
    "X_t = []\n",
    "y_train = []\n",
    "for i, token in enumerate(train_features): \n",
    "    X_t.append(token.input_ids)\n",
    "    #print(token.label_id)\n",
    "    y_train.append(formOfList_label_list_mult[token.label_id])\n",
    "X_t = np.asarray(X_t)\n",
    "y_train = np.asarray(y_train)\n",
    "print('loaded')\n",
    "\n",
    "print('loading X_te')\n",
    "X_te = []\n",
    "for i, token in enumerate(test_features): \n",
    "    X_te.append(token.input_ids)\n",
    "    #print(token.label_id)\n",
    "X_te = np.asarray(X_te)\n",
    "print('X_te loaded')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing torch bert  model...\n",
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n",
      "embedding matrix defined\n",
      "model defined and compiled \n",
      "Train on 143613 samples, validate on 15958 samples\n",
      "Epoch 1/2\n",
      "  6368/143613 [>.............................] - ETA: 1:04:10 - loss: 0.1772 - acc: 0.9570"
     ]
    }
   ],
   "source": [
    "\n",
    "# In[19]:\n",
    "\n",
    "print('importing torch bert  model...')\n",
    "import torch\n",
    "from pytorch_pretrained_bert import BertModel, BertTokenizer, BertForSequenceClassification\n",
    "bert_model = BertModel.from_pretrained(\"bert-base-uncased\",cache_dir=\"model\")\n",
    "embedding_matrix = []\n",
    "for token in bert_model.embeddings.word_embeddings.parameters():\n",
    "    embedding_matrix.append(token)\n",
    "emb_ma = embedding_matrix[0].tolist()\n",
    "array_emb_ma = np.asarray(emb_ma)\n",
    "\n",
    "print('embedding matrix defined')\n",
    "embedding_matrix = array_emb_ma\n",
    "\n",
    "\n",
    "# Simple bidirectional LSTM with two fully connected layers. We add some dropout to the LSTM since even 2 epochs is enough to overfit.\n",
    "\n",
    "# In[21]\n",
    "\n",
    "\n",
    "# In[22]:\n",
    "\n",
    "\n",
    "inp = Input(shape=(maxlen,))\n",
    "\n",
    "x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
    "\n",
    "x = Bidirectional(LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "x = GlobalMaxPool1D()(x)\n",
    "x = Dense(50, activation=\"relu\")(x)\n",
    "x = Dropout(0.1)(x)\n",
    "x = Dense(6, activation=\"sigmoid\")(x)\n",
    "model = Model(inputs=inp, outputs=x)\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "print('model defined and compiled ')\n",
    "# Now we're ready to fit out model! Use `validation_split` when not submitting.\n",
    "\n",
    "\n",
    "model.fit(X_t, y_train, batch_size=32, epochs=2, validation_split=0.1);\n",
    "\n",
    "\n",
    "# And finally, get predictions for the test set and prepare a submission CSV:\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "y_test = model.predict([X_te], batch_size=1024, verbose=1)\n",
    "sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')\n",
    "sample_submission[list_classes] = y_test\n",
    "sample_submission.to_csv('submission.csv', index=False)\n",
    "\n",
    "# serialize model to YAML\n",
    "model_yaml = model.to_yaml()\n",
    "with open(\"model.yaml\", \"w\") as yaml_file:\n",
    "    yaml_file.write(model_yaml)\n",
    "# serialize weights to HDF5\n",
    "model.save_weights(\"model.h5\")\n",
    "print(\"Saved model to disk\")\n",
    "\n",
    "\n",
    "# In[12]:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
